<!DOCTYPE html>
<html>
    <head>
        <script>
            var oldTime = Date.now();
            setInterval(function () {
                if((document.body.scrollHeight - document.body.scrollTop) < window.screen.height*1.5)
                    document.body.scrollTop = document.body.scrollHeight;
                if (document.body.scrollTop == document.body.scrollHeight) {
                } else {
                    document.getElementById('time').innerText = Date.now() - oldTime;
                }
            }, 10);
        </script>
        <style>
            #panel{width:600px; height:600px;position: fixed; top: 100px; right: 40px;font-size: 30px;border:1px solid #333;border-radius: 5px;}
        </style>
    </head>
    <body>
        <div id="panel"><span id="time"></span></div>
    </body>
</html>

<?php

/**
 *  使用方法:
  $url = 'http://www.qq.com/';
  单页抓取
  $bot = new botFullwebpage();
  $bot->onePage($url);

  整站抓取
  $bot = new botFullwebpage();
  $bot->webMap($url);
  foreach ($bot->urls as $url) {
  $bot->onePage($url);
  }
 * 
 * 参数说明：
 * $patternDownFile         匹配下载文件
 * $patternPageUrl          匹配网站地图的url
 * $urlFilter               过滤url内的关键字：作用于减少抓取不必要的文件，[ad]这个关键字会被一些路由器当广告过滤
 * $downFile                需要下载的文件：爬取页面后，需要下载的文件类型（包含追踪文件类型）
 * $downSolidityFile		需要下载的文件：爬取页面后，需要下载的文件类型（不包含需要继续追踪的文件类型）
 * $imgFile					图片文件
 * $cachePath               缓存目录：存储在本地的缓存目录，默认存储24小时
 * $rootPath                根目录：存储在本地的根目录
 * $urls                    url集合：用于存储爬取网站地图的url结果
 * $firestWattingUrlTime    每次开始发起请求所等待的时间
 */
class botFullWebpage {

    public $requestUrl = [];
    public $url = '';
    public $mainUrlVars = [];
    public $patternDownFile = '';
    public $patternPageUrl = '';
    public $urlFilter = ['baidu', 'sina', 'google-analytics', 'googlesyndication', 'google', '.exe']; //, 'yahoo', '163', 'cdn', 'gravatar', 'country-select.min.js', 'address-i18n.min.js', 'frontend/cart.min.js', 'stats.g.doubleclick.net/dc.js', 's/devicepx-jetpack.js'); //, 'ad', '{', '(', '\\'); // filter keyword
    public $downFile = ['css', 'js', 'jpg', 'jpeg', 'png', 'bmp', 'ico', 'gif', 'svg', 'ttf', 'font', 'flv', 'swf', 'woff', 'woff2', 'eot'];
    public $downSolidityFile = ['jpg', 'jpeg', 'png', 'bmp', 'ico', 'gif', 'svg', 'ttf', 'font', 'flv', 'swf', 'woff', 'woff2', 'eot']; // download solid state object keyword
    public $imgFile = ['jpg', 'jpeg', 'png', 'bmp', 'ico', 'gif', 'svg'];
    public $domain = '';
    public $cachePath = './cache/';
    public $rootPath = './bot-template/';
    public $urls = [];
    public $firestWattingUrlTime = 10000;

    public function init($url) {
        $this->url = $url;
        $this->mainUrlVars = parse_url($url);
        $this->domain = $this->mainUrlVars['scheme'] . '://' . $this->mainUrlVars['host'];
        $this->patternDownFile = '/(\'|\"|\/\/|\()([^\s\'\"\=\(\*\#\<\>]+\.(' . implode('|', $this->downFile) . '))[\'\"\?\)]/i';
        $this->patternPageUrl = '/(src|href)\=[\s\'\"]+([^\s\'\"\#\<\>]+)[\s\'\"\#]/i'; //'/http\:\/\/(' . $this->mainUrlVars['host'] . '|\.\/)[^\s\'\"\)\(]+\/[^\s\'\"]+/i';
    }

    public function webMap($url, $label = '') {
        $this->init($url);
        $body = $this->getHtml($url, empty($label) ? '' : $label);
        preg_match_all($this->patternPageUrl, $body, $result);
        $urlResult = $result[2];
        $urlResult = array_filter($urlResult);
        $urlResult = array_unique($urlResult);
        foreach ($urlResult as $oldUrl) {
            if (TRUE === $this->isFilterValueInUrl($oldUrl)) {
                continue;
            }
            $newUrl = $this->repairUrl(stripslashes(trim($oldUrl)), $url);

            $parse = parse_url($newUrl);
            if (strlen(str_replace($this->filterFileEx, '', $newUrl)) === strlen($newUrl) && false !== strpos($newUrl, '/') && strpos($newUrl, $this->mainUrlVars['host']) && !empty($parse['path']) && (strlen($parse['path']) > 3)) {
                if (empty($this->urls[md5($newUrl)])) {
                    $this->urls[md5($newUrl)] = $newUrl;
                }
            }
        }
    }

    public function onePage($url) {
        $this->init($url);
        echo "###### $url ######";
        $html = $this->go($this->url);
        file_put_contents($this->rootPath . implode('', explode('.', $this->mainUrlVars['host'])) . '/' . md5($url) . '.html', $html);
        echo PHP_EOL . '<br>ok<br>' . PHP_EOL;
    }
    
    public function relativityCSSPath($path, $url) {
        $path = preg_replace('/\\|\/+/', '/', $path);
        $extension = $this->getUrlFileType($path);
        if (strpos($path, 'http://') === 0 || strpos($path, 'https://') === 0) {
            $parseUrl = parse_url($url);
            return str_replace('//', '/', '../' . $parseUrl['host'] . '/' . $extension . '/' . md5($path)) . '.' . $extension;
        } else {
            $parseUrl = parse_url($url);
            return str_replace('//', '/', '../' . $extension . '/' . md5($path)) . '.' . $extension;
        }
    }

    public function repairUrl($oldUrl, $url) {
        $newUrl = $newPath = '';
        $urlAvgs = parse_url($url);
        $domain = $urlAvgs['scheme'] . '://' . $urlAvgs['host'];

        if (0 !== strpos($oldUrl, 'http')) {
            if (0 === strpos($oldUrl, '//')) {
                $newUrl = $urlAvgs['scheme'] . ':' . $oldUrl;
            } elseif (0 === strpos($oldUrl, '/')) {
                $newUrl = $domain . $oldUrl;
            } elseif (0 === strpos($oldUrl, '../')) {
                $tmpPath = explode('../', $oldUrl);
                $newUrl = $url;
                while ($tmpPath) {
                    $newUrl = $this->dirname($newUrl, $domain);
                    $newPath = array_shift($tmpPath);
                }
                $newUrl = ($newUrl[0] == '/' || strlen($newUrl) < strlen($domain)) ? $domain . $newUrl . $newPath : $newUrl . $newPath;
            } else {
                $newUrl = $this->dirname($url, $domain) . $oldUrl;
            }
        } else {
            $newUrl = $oldUrl;
        }
        return $newUrl;
    }

    public function htmlFormatHash($body) {
        $hash = md5(preg_replace(array('/\s/', '/>[^<]+</'), array('', '><'), $body));
        return $this->getCache($hash);
    }

    //抓取资源
    public function go($url, $label = '') {
        var_dump('current url:' . $url);
        $result = array();
        $body = $this->getHtml($url, empty($label) ? '' : $label);

//        if ($this->htmlFormatHash($body)) {//剔除重复的html结构页面，避免多次采集
//            return FALSE;
//        }

        preg_match_all($this->patternDownFile, $body, $result);
//        print_r($result);die;

        $urlResult = $result[2];
        $urlResult = array_filter($urlResult);
        $urlResult = array_unique($urlResult);
//        echo '<pre>'; print_r($urlResult);

        $host = str_replace('.', '', $this->mainUrlVars['host']);
        foreach ($urlResult as $oldUrl) {
            var_dump('parent url ### ' . $url);
            var_dump('bot url ### ' . $oldUrl);

            if (TRUE === $this->isFilterValueInUrl($oldUrl)) {
                continue;
            }

            $newUrl = $this->repairUrl(stripslashes(trim($oldUrl)), $url);
            if (FALSE === strpos($newUrl, 'http')) {
                die('error--' . $newUrl);
            }

            $urlParse = parse_url($newUrl);
            $urlParse['path'] = preg_replace('/\/+/', '/', $urlParse['path']);
            $urlParse['host'] = empty($urlParse['host']) ? '' : str_replace(['\\', '.'], ['/', ''], $urlParse['host']);
//            $urlParse['query'] = empty($urlParse['query']) ? '' : $urlParse['query'];

            $path = '/' . $this->getUrlFileType($oldUrl) . '/' . md5($oldUrl) . '.' . $this->getUrlFileType($oldUrl);

            if ($urlParse['host'] != $host) {
                $path = '/' . $urlParse['host'] . $path;
            }

            if ($this->getUrlFileType($url) == 'css') { // css
                $cssPath = $this->relativityCSSPath($oldUrl, $url);
                var_dump('css ### ' . $cssPath);
                $body = str_replace($oldUrl, $cssPath, $body);
            } elseif ($this->getUrlFileType($url) == 'js') { // css
            } else {
                var_dump('html ### ' . $path);
                $body = str_replace($oldUrl, '.' . $path, $body);
            }

            $prefix = $this->rootPath . $host;
            $saveFilePath = $prefix . $path;

            if (!is_dir(dirname($saveFilePath))) {
                mkdir(dirname($saveFilePath), 0755, 1);
            }

            var_dump('down file path #### ' . $saveFilePath);

            if ($this->isSolidityFile($newUrl)) {
                $content = $this->getHtml($newUrl, empty($urlParse['scheme']) ? '' : $urlParse['scheme']);
                $content && file_put_contents($saveFilePath, $content);
            } else {
                $content = $this->go($newUrl, empty($urlParse['scheme']) ? '' : $urlParse['scheme']);
                $content && file_put_contents($saveFilePath, $content);
            }
        }

        return $this->filterHtmlBody($body);
    }

    public function dirname($path, $domain = '') {
        if (in_array($path, array('.', '/', '\\', '#', '?', ''))) {
            return '/';
        } elseif ('/' == $path[0]) {
            $tmp = explode('/', $path);
            return count($tmp) > 2 ? str_replace('\\', '', dirname($path)) . '/' : '/';
        } elseif (!empty($domain)) {
            if (strlen($domain) <= strlen(dirname($path))) {
                return str_replace('\\', '', dirname($path)) . '/';
            }
        }
        return $domain;
    }

    public function getUrlFileType($url) {
        foreach ($this->downFile as $v) {
            if (strpos($url, '.' . $v) !== false) {
                return $v;
            }
        }
        return 'html';
    }

    public function isSolidityFile($file) {
        foreach ($this->downSolidityFile as $v) {
            if (strpos($file, '.' . $v) !== false) {
                return $v;
            }
        }
        return FALSE;
    }

    public function isFilterValueInUrl($str) {
        if (empty($str)) {
            return FALSE;
        }
        foreach ($this->urlFilter as $v) {
            if (FALSE !== strpos($str, $v)) {
                return TRUE;
            }
        }
        return FALSE;
    }

    public function filterHtmlBody($body) {
        $find = array('/<base[^>]+>/i', '/<link rel\="dns\-prefetch"[^>]+>/i');
        $replace = array('', '');
        $body = preg_replace($find, $replace, $body);
        $find = array($this->domain);
        $replace = array('.');
        return str_replace($find, $replace, $body);
    }

    public function getHtml($url, $label = '') {
        echo PHP_EOL . '<br><i style="color:green;">--START:</i>' . $url . PHP_EOL;
        echo str_repeat(" ", 4096); //确保足够的字符，立即输出，Linux服务器可以去掉这个语句
        $matches = array();
        $headerParam = empty($this->cookie) ? array('CURLOPT_HEADER' => 1) : array('CURLOPT_HEADER' => 1, 'CURLOPT_COOKIE' => $this->cookie);
        $content = $this->getCache(md5($url)) ? $this->getCache(md5($url)) : $this->visitUrl($url, $headerParam);
        // 解析HTTP数据流
        $split = explode("\r\n\r\n", $content);
        $header = array_shift($split);
        $body = implode("\r\n\r\n", $split);

        // 解析COOKIE
        if (preg_match("/set\-cookie:([^\r\n]*)/i", $header, $matches)) {
            $this->cookie = $matches[1];
        }
        if (empty($body)) {
            $this->requestUrl[md5($url)] = 1;
            echo PHP_EOL . '<br>--<i style="color:red;">ERROR:</i>' . $url . PHP_EOL;
            if ('http' == $label) {
                return $this->getHtml(str_replace('http', 'https', $url));
            } elseif ('https' == $label) {
                return $this->getHtml(str_replace('https', 'http', $url));
            }
        } else {
            $this->requestUrl[md5($url)] = 1;
            echo PHP_EOL . '<br>----<i style="color:greenyellow;">END:</i>' . $url . PHP_EOL;
            file_put_contents($this->cachePath . md5($url), $content);
        }
        return $body;
    }

    function getCache($hashstr) {
        clearstatcache();
        if (!is_dir($this->cachePath)) {
            mkdir($this->cachePath, 0755, 1);
        }
        $file = $this->cachePath . $hashstr;
        if (is_file($file)) {
            $fTime = (int) filemtime($file);
            if (($fTime + 86400) > time()) {
                return file_get_contents($file);
            }
        }
        return FALSE;
    }

    public function visitUrl($url, $param = array()) {
//        if(!empty($this->requestUrl[md5($url)]) && $this->requestUrl[md5($url)] == 1){
//            return '';
//        }
        $urlParse = parse_url($url);
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER['HTTP_USER_AGENT']); //"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.1) Gecko/20061204 Firefox/2.0.0.1");
        curl_setopt($ch, CURLOPT_HEADER, empty($param['CURLOPT_HEADER']) ? 0 : $param['CURLOPT_HEADER']);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
        curl_setopt($ch, CURLOPT_AUTOREFERER, 1); // auto setting Referer
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); // auto just
        curl_setopt($ch, CURLOPT_TIMEOUT_MS, $this->firestWattingUrlTime); // watting firest visit url time
//        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT_MS, 3000); 
        curl_setopt($ch, CURLOPT_DNS_CACHE_TIMEOUT, 300);
        if (!empty($param['CURLOPT_COOKIE'])) {
            curl_setopt($ch, CURLOPT_COOKIE, $param['CURLOPT_COOKIE']);
        }
        if ($urlParse['scheme'] == 'https') {
//            $sslFile = getcwd() . '/GoDaddyRootCertificateAuthority-G2.crt';//'/lfatabletpresses_d.cer';//'/cacert.pem';//
//            if(!is_file($sslFile)){
//                echo 'ERROR: https not file:'.$sslFile;
//            }
            curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0);
//            curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2);
//            curl_setopt($ch, CURLOPT_CAINFO, $sslFile);
        }
        if (!empty($param['CURLOPT_POSTFIELDS'])) {
            curl_setopt($curl, CURLOPT_POST, 1); // start post
            curl_setopt($curl, CURLOPT_POSTFIELDS, $param['CURLOPT_POSTFIELDS']); // Post data
        }
        $content = curl_exec($ch);
//        var_dump($content);
//        var_dump(curl_getinfo($ch, CURLINFO_HTTP_CODE));
//        die;
        if (curl_getinfo($ch, CURLINFO_HTTP_CODE) !== 200) {
            return '';
        }
        curl_close($ch);
        return $content;
    }

}
